package org.tony.web.spider;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;

public class JsoupDemo
{

  /**
   *    Document doc = Jsoup.connect("http://www.baidu.com").userAgent("Mozilla/5.0")
        .header("Accept-Charset", "utf-8").get();
    System.out.println(doc);
   * @param args
   * @throws IOException 
   */
  public static void main(String[] args) throws IOException
  {
    System.setProperty("http.proxyHost", "10.32.235.41");
    System.setProperty("http.proxyPort", "8080");
    //System.out.println(Jsoup.connect("http://tieba.baidu.com/p/1765510247").userAgent("Mozilla/5.0").get());
    System.out.println(getContent("http://tieba.baidu.com/p/1765510247"));
  }

  public static String getContent(String strUrl)
  {

    try
    {

      URL url = new URL(strUrl);

      BufferedReader br = new BufferedReader(new InputStreamReader(url

      .openStream()));

      String s = "";

      StringBuffer sb = new StringBuffer("");

      while ((s = br.readLine()) != null)
      {

        sb.append(s + "/r/n");

      }

      br.close();

      return sb.toString();

    }
    catch (Exception e)
    {

      return "error open url:" + strUrl;

    }

  }
}
